import feapder
from feapder import Item

from typing import TypeVar, Generic, List


class BookCategory(object):
    """
    书籍分类对象
    """

    # 使用装饰器模式，会自动添加一个type_one属性。当调用该属性时，自动调用该装饰的方法
    def __init__(self, url: str, type_one: str, type_two: str, type_three: str):
        self.__url = url
        self.__type_one = type_one
        self.__type_two = type_two
        self.__type_three = type_three

    def __str__(self):
        return "url:" + self.__url + ", type_one:" + self.__type_one + ", type_two: " + self.__type_two \
               + ", type_three: " + self.__type_three

    @property
    def url(self):
        return self.__url

    @url.setter
    def url(self, value):
        if not isinstance(value, str):
            raise ValueError('url must be a str!')
        self.__url = value

    @property
    def type_one(self):
        return self.__type_one

    @type_one.setter
    def type_one(self, value):
        if not isinstance(value, str):
            raise ValueError('type_one must be a str!')
        self.__type_one = value

    @property
    def type_two(self):
        return self.__type_two

    @type_two.setter
    def type_two(self, value):
        if not isinstance(value, str):
            raise ValueError('type_two must be a str!')
        self.__type_two = value

    @property
    def type_three(self):
        return self.__type_three

    @type_three.setter
    def type_three(self, value):
        if not isinstance(value, str):
            raise ValueError('type_three must be a str!')
        self.__type_three = value


class DangDangBook(object):
    """
    书籍对象
    """

    def __init__(self, num: int, details_url: str, img_url: str, tuijian: str, publisher_author_nationality: str,
                 publisher_author_name: str, publisher: str, prce: str, title_descript: str, descript: str):
        self.num = num
        self.details_url = details_url
        self.img_url = img_url
        self.tuijian = tuijian
        self.publisher_author_nationality = publisher_author_nationality
        self.publisher_author_name = publisher_author_name
        self.publisher = publisher
        self.prce = prce
        self.title_descript = title_descript
        self.descript = descript

    def set_title_descript(self, val):
        self.title_descript = val

    def set_descript(self, val):
        self.descript = val


Vector = list[BookCategory]


class Spider(feapder.AirSpider):
    __custom_setting__ = dict(
        ITEM_PIPELINES=["feapder.pipelines.mongo_pipeline.MongoPipeline"],
        MONGO_IP="server.windows",
        MONGO_PORT=27017,
        MONGO_DB="feapder",
        MONGO_USER_NAME="root",
        MONGO_USER_PASS="141535",
    )

    # 初始任务下发入口
    def start_requests(self):
        # 请求
        yield feapder.Request("http://book.dangdang.com/")

    # 解析结果
    def parse(self, request, response):
        result = Vector()
        div_base = response.xpath('//div[@class="con flq_body"][1]')
        list_level_one = div_base.xpath('./div[@class="level_one "]')

        for level_one in list_level_one:
            dt = level_one.xpath('./dl[@class="primary_dl" or @class="con primary_dl"][1]/dt[1]')
            list_child_dt = dt.xpath('./child::a')

            if len(list_child_dt) == 0:
                type_one = dt.xpath('./text()').extract_first()
                if type_one is not None:
                    type_one = type_one.strip()
                    result_typetwo = self.getTypeTwo(level_one)
                    _result_typetwo = result_typetwo["result"]
                    for r in _result_typetwo:
                        result.append(BookCategory(r["href"], type_one, r["typetwo"], r["typethree"]))
            else:
                for a in list_child_dt:
                    type_one = a.xpath('./text()').extract_first()
                    if type_one is not None:
                        type_one = type_one.strip().replace("/", "")
                        result_typetwo = self.getTypeTwo(level_one)
                        _result_typetwo = result_typetwo["result"]
                        for r in _result_typetwo:
                            result.append(BookCategory(r["href"], type_one, r["typetwo"], r["typethree"]))
        for r in result:
            item = Item()  # 声明一个item
            item.table_name = "book_category"  # 指定存储的表名
            item.url = r.url  # 给item属性赋值
            item.type_one = r.type_one
            item.type_two = r.type_two
            item.type_three = r.type_three
            yield item  # 返回item， item会自动批量入库

    @staticmethod
    def getTypeTwo(level_one):
        """
        获取typetwo相关信息
        :param level_one: level_one
        :return: {"result": result, "result_typetwo": result_typetwo}
        """
        list_left_dl = level_one.xpath('.//div[@class="hide submenu "]//dl[@class="inner_dl"]')
        # 普通结果集合
        result = []
        # 二级分类可点击。
        result_typetwo = []
        if list_left_dl is not None:
            for dl in list_left_dl:
                # 二级分类名称
                typetwo = None
                # 二级分类如果可以点击的链接
                url_typetwo = None
                dt = dl.xpath("./dt[1]")
                a_dt = dt.xpath('./child::a[1]')
                # dt有两种情况。1 无法点击；2 可点击，有子标签<a>
                if len(a_dt) == 0:
                    typetwo = dt.xpath('./text()').extract_first()
                else:
                    typetwo = a_dt.xpath('./text()').extract_first()
                    href = a_dt.xpath('./@href').extract_first()
                    if len(href) != 0:
                        href = href.strip()
                    result_typetwo.append({'typetwo': typetwo, 'href': href})
                if typetwo is not None:
                    typetwo = typetwo.strip()
                    list_a = dl.xpath("./dd[1]/a")
                    for a in list_a:
                        href = a.xpath("./@href").extract_first()
                        text = a.xpath("./text()").extract_first()
                        result.append({'typetwo': typetwo, 'href': href, 'typethree': text})
        return {"result": result, "result_typetwo": result_typetwo}


if __name__ == "__main__":
    Spider().start()
